
# Load up our libraries
library(tidyverse)


dino <- ">DinoDNA from Crichton JURASSIC PARK  p. 103 nt 1-1200 GCGTTGCTGGCGTTTTTCCATAGGCTCCGCCCCCCTGACGAGCATCACAAAAATCGACGCGGTGGCGAAACCCGACAGGACTATAAAGATACCAGGCGTTTCCCCCTGGAAGCTCCCTCGTGTTCCGACCCTGCCGCTTACCGGATACCTGTCCGCCTTTCTCCCTTCGGGAAGCGTGGCTGCTCACGCTGTACCTATCTCAGTTCGGTGTAGGTCGTTCGCTCCAAGCTGGGCTGTGTGCCGTTCAGCCCGACCGCTGCGCCTTATCCGGTAACTATCGTCTTGAGTCCAACCCGGTAAAGTAGGACAGGTGCCGGCAGCGCTCTGGGTCATTTTCGGCGAGGACCGCTTTCGCTGGAGATCGGCCTGTCGCTTGCGGTATTCGGAATCTTGCACGCCCTCGCTCAAGCCTTCGTCACTCCAAACGTTTCGGCGAGAAGCAGGCCATTATCGCCGGCATGGCGGCCGACGCGCTGGGCTGGCGTTCGCGACGCGAGGCTGGATGGCCTTCCCCATTATGATTCTTCTCGCTTCCGGCGGCCCGCGTTGCAGGCCATGCTGTCCAGGCAGGTAGATGACGACCATCAGGGACAGCTTCAACGGCTCTTACCAGCCTAACTTCGATCACTGGACCGCTGATCGTCACGGCGATTTATGCCGCACATGGACGCGTTGCTGGCGTTTTTCCATAGGCTCCGCCCCCCTGACGAGCATCACAAACAAGTCAGAGGTGGCGAAACCCGACAGGACTATAAAGATACCAGGCGTTTCCCCCTGGAAGCGCTCTCCTGTTCCGACCCTGCCGCTTACCGGATACCTGTCCGCCTTTCTCCCTTCGGGCTTTCTCAATGCTCACGCTGTAGGTATCTCAGTTCGGTGTAGGTCGTTCGCTCCAAGCTGACGAACCCCCCGTTCAGCCCGACCGCTGCGCCTTATCCGGTAACTATCGTCTTGAGTCCAACACGACTTAACGGGTTGGCATGGATTGTAGGCGCCGCCCTATACCTTGTCTGCCTCCCCGCGGTGCATGGAGCCGGGCCACCTCGACCTGAATGGAAGCCGGCGGCACCTCGCTAACGGCCAAGAATTGGAGCCAATCAATTCTTGCGGAGAACTGTGAATGCGCAAACCAACCCTTGGCCATCGCGTCCGCCATCTCCAGCAGCCGCACGCGGCGCATCTCGGGCAGCGTTGGGTCCT"


...(string = dino, ...)


str_remove_all(string = dino, pattern = ... )


str_remove(string = dino, ... ) # remove entire header

# removes > and D, the first two characters in the string


str_remove(string = dino, pattern = ...) # lazy matching removes the first occurrence of a group of digits followed by a space


str_remove(string = dino, pattern = ... ) # Greedy matching catches everything between > and the very last occurrence of a group of digits followed by a space. The first digit-space group is now part of the "everything in between" part of the expression


dna <- ...
dna


str_extract(string = dino, pattern = ...)

#header <- ...
#header


# percentage of GC content


str_replace_all(dna, 
                c("G" = "", # placeholder to prevent all C from becoming G
                  "C" = "", 
                  "A" = "", 
                  "T" = "",
                  "X" = ""))

#mrna <- 
#mrna


str_count(...)


# try out string locate on mrna


# vs string locate all on mrna


# Subsetting from position 90 (beginning of start codon) to base 1,200 (last base in our sequence). 
# Note that the beginning of the sequence changed
...

#is equivalent to
... # Look at the nested code (one function inside another)

#mrna <-
#mrna


# think of these dots as wildcards to pick any character triplets

# how many codons did we retrieve?
length()


codons <- ...(str_extract_all(mrna, "..."))

codons

length()


...(str_detect(codons, "(UAG)|(UGA)|(UAA)") == TRUE) # Which returns a non-boolean

# Note that this is equivalent
 # Output is boolean


codons[1:98]

#equivalent to 
#translation <- ... # The search will be stopped at the first detection of either of the stop codons listed

#translation


...(translation)

codon_translation = c("UUU" = "F", "UCU" = "S", "UAU" = "Y", "UGU" = "C", 
                      "UUC" = "F", "UCC" = "S", "UAC" = "Y", "UGC" = "C", 
                      "UUA" = "L", "UCA" = "S", "UAA" = "*", "UGA" = "*", 
                      "UUG" = "L", "UCG" = "S", "UAG" = "*", "UGG" = "W", 
                      "CUU" = "L", "CCU" = "P", "CAU" = "H", "CGU" = "R", 
                      "CUC" = "L", "CCC" = "P", "CAC" = "H", "CGC" = "R", 
                      "CUA" = "L", "CCA" = "P", "CAA" = "Q", "CGA" = "R", 
                      "CUG" = "L", "CCG" = "P", "CAG" = "Q", "CGG" = "R", 
                      "AUU" = "I", "ACU" = "T", "AAU" = "N", "AGU" = "S", 
                      "AUC" = "I", "ACC" = "T", "AAC" = "N", "AGC" = "S", 
                      "AUA" = "I", "ACA" = "T", "AAA" = "K", "AGA" = "R", 
                      "AUG" = "M", "ACG" = "T", "AAG" = "K", "AGG" = "R", 
                      "GUU" = "V", "GCU" = "A", "GAU" = "D", "GGU" = "G", 
                      "GUC" = "V", "GCC" = "A", "GAC" = "D", "GGC" = "G", 
                      "GUA" = "V", "GCA" = "A", "GAA" = "E", "GGA" = "G", 
                      "GUG" = "V", "GCG" = "A", "GAG" = "E", "GGG" = "G")


# translation <- ...

# translation


str_flatten(translation)

# Equivalent call using the paste function
paste()

translation <- str_flatten(translation)


# recombine header with our translation and separate with a space
str_c(...)


getwd()
#list.files()

# read file as a single string
#fasta <- 
#fasta # a single continuous string with all genes. Show students the line break characters


# Retrieve the structure 


# is it a vector?


# What are it's dimensions?
 # Is unidimensional so the output of dim() is NULL


# what is we use print?
print()


writeLines() # Takes into account line breaks


cat() # Another base-r alternative


# string count through our object for the ">" symbol. How many entries do we have?


fasta  ...(..., # look for opening or closing squared brackets
                           replacement = "")  #paste("\\1", "\\2", sep = " ")) # replace squared brackets with nothing


# Use str_split to break up the line breaks
fasta  %>% 
str_replace_all(pattern = "\\[|\\]", replacement = "")  %>% # Remove square brackets
... %>% 
head()


# You need to convert the result to a vector to work with.
fasta  %>% 
str_replace_all(pattern = "\\[|\\]", replacement = "")  %>% # Remove square brackets
str_split("\r\n\r\n")  %>% # Use str_split to break up the line breaks
... %>% 
head()


# remove the ">" from the start of each entry
fasta  %>% 
str_replace_all(pattern = "\\[|\\]", replacement = "")  %>% # Remove square brackets
str_split("\r\n\r\n")  %>% # Use str_split to break up the line breaks
unlist()  %>% # You need to unlist the result to work with easier
...  %>% 
head()


# Break each entry into 2 columns with str_split_fixed and save it
header_seq_fasta <- fasta %>% 
str_replace_all(pattern = "\\[|\\]", replacement = "")  %>% # Remove square brackets
str_split("\r\n\r\n")  %>% # Use str_split to break up the line breaks
unlist()  %>% # You need to unlist the result to work with easier
str_replace_all(pattern = "^>", replacement = "")  %>% # remove the ">" from the start of each entry
...  %>% 
...


head(header_seq_fasta)


# Which entries within our dataset are duplicated by sequence?


# Use which to filter out duplicated copies
header_seq_subset <- header_seq_fasta[...]
str(header_seq_subset)


# Use filter to remove duplicated entries
header_seq_subset <- header_seq_fasta %>% ...

str(header_seq_subset)


# What pattern will we use to break down our header into components?
fasta_header <- str_match_all(...) # this is a list

print(tail(fasta_header, 3))


# Look at the first entry of fasta_header

# Two ways to drop the first column


# Convert the results of str_match_all into a data frame
data.frame()

# You need to transpose the trimmed data for proper coercion


# What happens if we call rbind?
rbind(fasta_header[[1]][,-1], )

# What about rbind.data.frame?


# Create a data frame and place-hold the column names to match with what we have in fasta6
header.df <- ...(fasta_header[[1]][,-1])

rm(i) ## remove any previous instances of i

for (i in 2:(length(fasta_header)-1)){

    header.df <- rbind.data.frame(...)
}

head(header.df)
str(header.df)


# Let's see what happens if we use lapply first
lapply(...)  %>%  head()


# Now use do.call
#header.df <-
do.call(...)  %>% 
select(-1) # remember you need to get rid of the first column


# rename the columns "Accession_number", "Protein_info", "Genus", and "Species"
colnames(header.df) <- ...

head(header.df,2)


# Confirm what our sequence data looks like
print(head(header_seq_subset[...]))


# Let's split off all of the extra line breaks
header_seq_subset[,2]  %>% 
...
# Now you'll have a list of split sequences that you'll need to re-unite

# Turn this result back into a vector

# drop the last element because it's not helpful


# Let's remove all of the extra line breaks
seq_united <- header_seq_subset[,2]  %>% 
... %>% 
# Now you'll have just have a vector that you need to trim
head(-1)


# we can combind the columns, but will we get the column names we want?
head(...)

# If we use dplyr and  %>%  we can keep passing this around for further use
# header.df %>% 
# ...  %>%  
# head()

# Add a new column to our data frame header.df but this is a permanent command
# header.df
# head(header.df)


writeLines("hello\n\U0001F30D") # U1F30D prints a planet icon on datacamp but not here
# similar to cat. Unicode characters in: http://www.unicode.org/charts/
cat("hello\n\U0001F30D")


# Use writeLines() with "\u0928\u092e\u0938\u094d\u0924\u0947 \u0926\u0941\u0928\u093f\u092f\u093e"
# Hello world in Hindi (taken from datacamp.com)
writeLines("\U00000928\u092e\u0938\u094d\u0924\u0947 \u0926\u0941\u0928\u093f\u092f\u093e")
cat("\u0928\u092e\u0938\u094d\u0924\u0947 \u0926\u0941\u0928\u093f\u092f\u093e")

Expression	Meaning
\w, [A-z, 0-9], [[:alnum:]]	word characters (letters + digits)
\d, [0-9], [[:digit:]]	digits
[A-z], [[:alpha:]]	alphabetical characters
\s, [[:space:]]	space
[[:punct:]]	punctuation
[[:lower:]]	lowercase
[[:upper:]]	uppercase
\W, [^A-z0-9]	not word characters
\S	not space
\D, [^0-9]	not digits

Expression	Meaning
?	0 or 1 occurrence
*	0 or more occurrences
+	1 or more occurrences
{n}	exactly n occurrences
{n,}	at least n occurrences
{,n}	at most n occurrences
{n,m}	between n and m occurrences (inclusive)

Expression	Meaning
\|	or
.	matches any single character
[ ... ]	matches ANY of the characters inside the brackets
[ ... - ... ]	matches a RANGE of characters inside the brackets
[ ^... ]	matches any character EXCEPT those inside the bracket
( ... )	grouping - used for [backreferencing] (https://www.regular-expressions.info/backref.html)

Expression	Meaning
^	start of the string
$	end of the string
\\b	empty string at either edge of the word
\\B	empty string that is NOT at the edge of a word

Expression	Meaning
^	Beginning of a line
>	Greater than symbol, used to designate the beginning of a new read in fasta format
\w	Match a word (alphabetic) character (Remember that a second escape character (backslash) is required)
+	Match the preceding metacharacter more than once
.	match a literal period
\d	Match a digit
\s	space
.+	anything, one or more times
\[	match a literal squared bracket
.*	anything, as many times it is present

Text Entry	Properties
XP_007980836.1	alphanumeric series, separated by ".", ending with digits
PREDICTED: forkhead box protein P2 isoform X2	alphanumeric with any number of words and spaces present
Chlorocebus	single alphabetical word
sabaeus	single alphabetical word

Lecture 05: Of Data Cleaning and Documentation - Conquer Regular Expressions and Challenge yourself with a 'Real' Dataset¶

0.1.0 A quick intro to the Introduction to R for Data Science¶

0.1.1 Class Objectives¶

0.2.0 How do we get there?¶

0.3.0 A legend for text format in Jupyter markdown¶

0.4.0 Data used in this session¶

0.4.1 Datafile 1: regex_word.docx¶

0.4.2 Datafile 2: FoxP2_primate.fasta¶

0.5.0 Packages Used in This Lesson¶

1.0.0 Data cleaning or "data munging" or "data wrangling"¶

2.0.0 Introduction to regular expressions (RegEx)¶

2.1.0 Classes¶

2.2.0 Quantifiers¶

2.3.0 Operators¶

2.4.0 Matching by position¶

2.5.0 Escape characters¶

2.6.0 Trouble-shooting RegEx¶

3.0.0 Online ReGex exercise¶

3.1.0 Breaking down a regular expression example¶

3.1.1 The breakdown:¶

3.2.0 Introduction to string manipulation with stringr¶

3.2.1 RStudio's find-replace¶

3.2.2 Remove unwanted text with str_remove() and str_remove_all()¶

3.2.2.1 str_remove() replaces the first instance of a search string versus str_remove_all()¶

3.2.3 Greedy vs. lazy matching¶

3.3.0 Extracting to save for later with str_extract()¶

3.4.0 Searching with str_extract_all()¶

3.5.0 Replacement using str_replace_all()¶

3.6.0 Useful search tools powered by RegEx¶

3.6.1 Simple questions with str_detect()¶

3.6.2 Counting pattern matches with str_count()¶

3.6.3 Locating pattern matches with str_locate()¶

3.7.0 Splitting up our string with str_sub()¶

3.8.0 Applying our RegEx skill further¶

3.8.1 Generate codons from the first open reading frame with a ...¶

3.8.1.1 Remember some functions return a list as output.¶

3.8.2 Search for multiple specific patterns using |¶

3.8.3 Identify specific occurrences in a vector with which()¶

3.8.4 Replace your codons with amino acids¶

3.8.5 Collapsing your results into a single string with str_flatten()¶

3.8.6 Recombining strings with str_c()¶

4.0.0 Multi FASTA file formatting¶

4.0.1 If you wanted to pull down your own version of the data set¶

4.1.0 Reorganizing a multi-entry fasta file¶

4.1.1 Read in your file as a single string using readr::read_file()¶

4.1.2 Explore your data to understand its structure¶

4.1.2.1 Use writelines() or cat() to see your output in a human-readable format¶

4.1.3 How many entries do we have?¶

4.2.0 Order of operations¶

4.2.1 Strip out and replace all occurences of [ and ]¶

4.2.2 Split each entry into an individual row¶

4.2.3 Remove the > from the start of each entry¶

4.2.4 Split your entries into a header and sequence with str_replace_all()¶

4.2.4.1 Remove duplicated() entries with the help of which()¶

4.2.4.1 Remove duplicated() entries with the help of filter()¶

4.2.5 Break up the header column into the components using str_match_all()¶

4.2.6 Reformat our header information into a data frame¶

4.2.6.1 A note about working with data frames and rbind()¶

4.2.6.2 Use a loop to build your data frame one row at a time¶

4.2.6.3 do.call() to save youself some trouble¶

4.2.7 Reformat our sequence entries with str_*() methods¶

4.2.8 Adding columns to your data frame - a tool for each purpose¶

5.0.0 Resources¶

5.1.0 Post-lesson assessment¶

6.0.0 Challenge: a real messy dataset¶

3.2.0 Introduction to string manipulation with `stringr`¶

3.2.2 Remove unwanted text with `str_remove()` and `str_remove_all()`¶

3.2.2.1 `str_remove()` replaces the first instance of a search string versus `str_remove_all()`¶

3.3.0 Extracting to save for later with `str_extract()`¶

3.4.0 Searching with `str_extract_all()`¶

3.5.0 Replacement using `str_replace_all()`¶

3.6.1 Simple questions with `str_detect()`¶

3.6.2 Counting pattern matches with `str_count()`¶

3.6.3 Locating pattern matches with `str_locate()`¶

3.7.0 Splitting up our string with `str_sub()`¶

3.8.1 Generate codons from the first open reading frame with a `...`¶

3.8.2 Search for multiple specific patterns using `|`¶

3.8.3 Identify specific occurrences in a vector with `which()`¶

3.8.5 Collapsing your results into a single string with `str_flatten()`¶

3.8.6 Recombining strings with `str_c()`¶

4.1.1 Read in your file as a single string using `readr::read_file()`¶

4.1.2.1 Use `writelines()` or `cat()` to see your output in a human-readable format¶

4.2.1 Strip out and replace all occurences of `[` and `]`¶

4.2.3 Remove the `>` from the start of each entry¶

4.2.4 Split your entries into a header and sequence with `str_replace_all()`¶

4.2.4.1 Remove `duplicated()` entries with the help of `which()`¶

4.2.4.1 Remove `duplicated()` entries with the help of `filter()`¶

4.2.5 Break up the header column into the components using `str_match_all()`¶

4.2.6.1 A note about working with data frames and `rbind()`¶

4.2.6.3 `do.call()` to save youself some trouble¶

4.2.7 Reformat our sequence entries with `str_*()` methods¶